# -*- coding:utf-8; -*-

import sqlite3
import urllib2
import htmllib
import os
import re

states = []

stateUrl = 'http://www.e-liuxue.com/search/list.php?nam=&state=%s&page=%d'
imgUrl = 'http://www.e-liuxue.com/search/dbimg/%s.gif'

p0 = re.compile("<a href='.*?'>")

con = None
cur = None
addr = None
f = None
flog = None
ranklist={}
keys=[ '校名',
        'rank',
        '地址',
        'SAT',
        'ACT',
        '推荐度',
        '大学信息',
        '美国大学地图',
        '体育运动',
        'ESL',
        '申请时重要事项',
        #'GPA',
        '大学特色',
        ]
def unescape(s):
    p = htmllib.HTMLParser(None)
    p.save_bgn()
    p.feed(s)
    return p.save_end()

def init():
    global f
    global flog
    for line in open('states.txt'):
        states.append(line.replace('\n',''))
    for line in open('rank.txt'):
        line=line.strip()
        if line:
            rank = line[:line.find('.')]
            name = line[line.find('.')+1:]
            rank=rank.strip()
            name=name.strip()
            ranklist[name]=rank
    if not os.path.exists('dbimg'):
        os.mkdir('dbimg')
    if not os.path.exists('htmls'):
        os.mkdir('htmls')
    f = open('result.html','w')
    f.write('<html><body><table>')
    f.write('\n<tr>')
    for index,key in enumerate(keys):
        f.write('\n<td style="border-style:solid;border-width:1px;">%s</td>' % key)
    f.write('\n</tr>')
    f.flush()
    flog = open('html.html','w')
    #if os.path.exists('list.db'):
    #    os.remove('list.db')
    #con = sqlite3.connect('list.db')
    #cur = con.cursor()
    #cur.execute('Create table list(name char)')
    #con.commit()
def getRank(name):
    if ranklist.has_key(name):
        return ranklist[name]
    else:
        return 'no rank'
def end():
    global f
    global flog
    f.write('</table></body></html')
    f.close()
    #flog.close()
def getURL(url):
    retry = 0
    while True:
        try:
            print 'getting %s' % url
            html = urllib2.urlopen(url).read()
            return html
        except Exception,e:
            if retry == 5:
                raise e
            else:
                retry += 1
def getStateImg(state):
    if os.path.exists('dbimg/%s.gif' % state):
        return
    url = imgUrl % state
    ff = open('dbimg/%s.gif' % state,'wb')
    ff.write(getURL(url))
    ff.close()
def getUniversities(state):
    universities = []
    page = 0
    while True:
        filename = 'htmls/%s%d.html' % (state,page)
        if not os.path.exists(filename):
            url = stateUrl % (state,page)
            html = getURL(url)
            open(filename ,'w').write(html)
        else:
            html=open(filename ,'r').read()
        table = html[html.find("<table class='collegelist'>"):]
        table = table[:table.find("</table>")]
        ul = p0.findall(table)
        ul = map(lambda x: 'http://www.e-liuxue.com/search/%s' % x.replace("<a href='",'').replace("'>",''), ul)
        
        for u in ul:
            universities.append(u)
        if not ul:
            break
        page += 1
    return universities

def getBlock(divider,html):
    data = html[:html.find(divider)]
    html = html[html.find(divider)+len(divider):]
    return data,html        
def getSAT(data):
    data=data.replace("<tr><td colspan='5'><img src='dbimg/SATscoresA.gif'></td></tr>",'')
    data=data.replace("<td width='14' rowspan='11'><img src='dbimg/SATscoresB.gif'></td>",'')
    data=data.replace("<td width='26' rowspan='11'><img src='dbimg/SATscoresD.gif'></td>",'')
    data=data.replace("<td width='15' rowspan='11'><img src='dbimg/SATscoresF.gif'></td>",'')
    data=data.replace("<tr><td width='120'><img src='dbimg/SATscoresG.gif'></td>",'')
    p=re.compile("<td width='120'><img src='dbimg/SATscoresG.gif'>.*?</tr>",re.M|re.S)
    data=p.sub('',data)
    #data=data.replace("<td width='120'><img src='dbimg/SATscoresG.gif'></td>\n\t\t      </tr>\n",'')
    data=data.replace("<td width='120'><img src='dbimg/SATscoresG.gif'></td>",'')
    
    satMath = {}
    satVerbal = {}
    data = data[data.find('<tr>'):]
    for i in range(6):
        tr = data[:data.find('</tr>')]
        data = data[data.find('</tr>')+5:]
        m=re.search("<td class='left' align='right'><img src='dbimg/fleshbar.png' height='16' width='(.*)' ></td>",tr)
        if m:
            satMath[i]=m.group(1)
        else:
            satMath[i]='0.0'
        m=re.search("<td class='right' align='left'><img src='dbimg/orangebar.png' height='16' width='(.*)' >",tr)
        if m:
            satVerbal[i]=m.group(1)
        else:
            satVerbal[i]='0.0'
    return [satMath,satVerbal]
    

def getACT(data):
    data=data.replace("<tr><td colspan='5'><img src='dbimg/ACTscoresA.gif'></td></tr>",'')
    data=data.replace("<td width='14' rowspan='11'><img src='dbimg/ACTscoresB.gif'></td>",'')
    data=data.replace("<td width='26' rowspan='11'><img src='dbimg/ACTscoresD.gif'></td>",'')
    data=data.replace("<td width='15' rowspan='11'><img src='dbimg/ACTscoresF.gif'></td>",'')
    data=data.replace("<tr><td width='120'><img src='dbimg/ACTscoresG.gif'></td>",'')
    p=re.compile("<td width='120'><img src='dbimg/ACTscoresG.gif'>.*?</tr>",re.M|re.S)
    data=p.sub('',data)
    #data=data.replace("<td width='120'><img src='dbimg/SATscoresG.gif'></td>\n\t\t      </tr>\n",'')
    data=data.replace("<td width='120'><img src='dbimg/ACTscoresG.gif'></td>",'')
    
    satMath = {}
    satVerbal = {}
    data = data[data.find('</tr>')+5:]
    data = data[data.find('<tr>'):]
    for i in range(5):
        tr = data[:data.find('</tr>')]
        data = data[data.find('</tr>')+5:]
        m=re.search("<td class='left' align='right'><img src='dbimg/fleshbar.png' height='16' width='(.*)' ></td>",tr)
        if m:
            satMath[i]=m.group(1)
        else:
            satMath[i]='0.0'
        m=re.search("<td class='right' align='left'><img src='dbimg/orangebar.png' height='16' width='(.*)' >",tr)
        if m:
            satVerbal[i]=m.group(1)
        else:
            satVerbal[i]='0.0'
    return [satMath,satVerbal]
def getUInfo(data):
    UInfo = {}
    ul,data = getBlock('<li>',data)
    ul,data = getBlock('<li>',data)
    UInfo['性质']=re.search("<span class='databasetext'>(.*)</span>",ul.replace('\n','')).group(1)
    ul,data = getBlock('<li>',data)
    UInfo['类别']=re.search("<span class='databasetext'>(.*)</span>",ul.replace('\n','')).group(1)
    ul,data = getBlock('<li>',data)
    UInfo['学期']=re.search("<span class='databasetext'>(.*)</span>",ul.replace('\n','')).group(1)
    ul,data = getBlock('<li>',data)
    m=re.search("<spanclass='databasetext'>&#32422;(.*)平方米</span>",ul.replace('\n','').replace('\t','').replace(' ',''))
    if m:
        UInfo['占地面积']=m.group(1)
    else:
        UInfo['占地面积']='no data'
    ul,data = getBlock('<li>',data)
    m=re.search("<spanclass='databasetext'>&#32422;(.*)人</span>",ul.replace('\n','').replace('\r','').replace('\t','').replace(' ',''))
    if m:
        UInfo['学生数']=m.group(1)
    else:
        UInfo['学生数']='no data'
    ul,data = getBlock('<li>',data)
    m=re.search("<spanclass='databasetext'>&#32422;(.*)美元</span>",ul.replace('\n','').replace('\r','').replace('\t','').replace(' ',''))
    if m:
        UInfo['学费']=m.group(1)
    else:
        UInfo['学费']='no data'
    ul,data = getBlock('<li>',data)
    m=re.search("<spanclass='databasetext'>&#32422;(.*)美元</span>",ul.replace('\n','').replace('\r','').replace('\t','').replace(' ',''))
    if m:
        UInfo['食宿费']=m.group(1)
    else:
        UInfo['食宿费']='no data'
    return UInfo
def getUniversityInfo(u):
    global addr
    global flog
    uid = re.search('id=(.*)&sys=',u).group(1)
    flog.write(uid + '\n')
    filename = 'htmls/%s.html'% uid
    if not os.path.exists(filename):
        url = u
        html = getURL(url)
        open(filename ,'w').write(html)
    else:
        html = open(filename,'r').read()
    
    html=html[html.find("<div id='box700'>"):]
    html=html[:html.find("<div class='adinfotop'>")]
    html=html.replace("<div class='clr'><span><img src='../image/spacer.gif' alt='' /></span></div>",'')
    html=html.replace("<div class='clr'><span><img src='image/spacer.gif' alt='' /></span></div>",'')
    html=html.replace("<div class='blue'><h3>ESL</h3></div>",'')
    html=html.replace('<br />','')
    html=html.replace('\t','')
    
    info = {}
    #nextdivider = "<div class='data'>"
    #data,html = getBlock(nextdivider,html)
    html = html[:html.find("<div class='blue-long'>")]
    #校名
    nextdivider = "<div class='data'>"
    data,html = getBlock(nextdivider,html)
    info['校名']=re.search("<h2 class='collegetitle'>(.*?)</h2>",data).group(1)
    print info['校名']
    info['rank']=getRank(info['校名'])
    info['地址']=re.search("<div class='collegeadress'>地址 : <span class='databasetext'>(.*?)</span></div>",data).group(1)
    
    #SAT    
    nextdivider = "<div class='data'>"
    data,html = getBlock(nextdivider,html)
    info['SAT'] = getSAT(data)
    #ACT
    nextdivider = "<div id='box320'>"
    data,html = getBlock(nextdivider,html)
    info['ACT'] = getACT(data)
    #SAKAE大学推荐度
    nextdivider = "<div class='data'>"
    data,html = getBlock(nextdivider,html)
    info['推荐度'] =re.search("<span class='osusumepoints'>(.*?)</span>",data).group(1) 
    #大学信息
    nextdivider = "<div class='data'>"
    data,html = getBlock(nextdivider,html)
    info['大学信息'] = getUInfo(data)
    #美国大学地图
    nextdivider = "<div  class='data'>"
    data,html = getBlock(nextdivider,html)
    info['美国大学地图'] =  addr
    
    #体育运动
    nextdivider = "<div class='data'>"
    data,html = getBlock(nextdivider,html)
    info['体育运动'] = re.sub('<.*?>','',data)
    #ESL
    nextdivider = "<div class='data'>"
    data,html = getBlock(nextdivider,html)
    info['ESL'] = re.search("<span class='databasetext'>(.*?)</span>",data.replace('\n','')).group(1)
    #申请时重要事项
    nextdivider = "<div class='data'>"
    data,html = getBlock(nextdivider,html)
    info['申请时重要事项'] = re.sub('<.*?>','',data)
    #GPA・TOEFL・SAT
    #data = html[:html.find("<div class='data'>")]
    #html = html[html.find("<div class='data'>")+18:]
    #info['GPA'] = re.sub('<.*?>','',data)
    #大学特色
    nextdivider = "<div class='data'>"
    data,html = getBlock(nextdivider,html)
    info['大学特色'] = re.sub('<.*?>','',data)
    
    return info
    
def pName(s):
    return s.replace('\t','').replace('\n','')
pRank=pName
pAddr=pName
def pInfo(s):
    return '\t'.join([s['性质'].replace('\t','').replace('\n','') \
                     ,s['类别'].replace('\t','').replace('\n','') \
                     ,s['学期'].replace('\t','').replace('\n','') \
                     ,s['占地面积'].replace('\t','').replace('\n','') \
                     ,s['学生数'].replace('\t','').replace('\n','') \
                     ,s['学费'].replace('\t','').replace('\n','') \
                     ,s['食宿费'].replace('\t','').replace('\n','') ])
    return "<table><tr><td>%s</td> \
                       <td>%s</td> \
                       <td>%s</td> \
                       <td>%s</td> \
                       <td>%s</td> \
                       <td>%s</td> \
                       <td>%s</td> \
                       </tr></table>" % \
           (s['性质'],s['类别'],s['学期'],s['占地面积'],s['学生数'],s['学费'],s['食宿费'],)
pRcmd=pName
pMap=pName
pSports=pName
pESL=pName
pIssue=pName
#pGPA=pName
pTese=pName
satRange = ['700~800','600~699','500~599','400~499','300~399','200~299',]
def pSAT(s):
    return '\t'.join([s[0][i].replace('\t','').replace('\n','') for i in range(6)]) +'\t' + '\t'.join([s[1][i].replace('\t','').replace('\n','') for i in range(6)])
    rv = '''		    <table cellspacing='0' cellpadding='0' align='center'>
		      <tr><td border="1px">math</td><td>range</td><td>verbal</td></tr>'''
    for i in range(6):
        rv=rv + '''<tr>
		        <td style="border-style:solid;border-width:1px" border="1px" align='right'>%0.2f</td>
                        <td style="border-style:solid;border-width:1px" border="1px" align='center'>%s</td>
			<td style="border-style:solid;border-width:1px" border="1px" align='left'>%0.2f</td>
		      </tr>
        ''' % (s[0][i],satRange[i],s[1][i])
    return rv + "</table>"
    rv='''		    <table class='sat' cellspacing='0' cellpadding='0' align='center'>
		      <tr><td colspan='5'><img src='dbimg/SATscoresA.gif'></td></tr>
		      <tr><td width='14' rowspan='11'><img src='dbimg/SATscoresB.gif'></td>
		        <td class='left' align='right'><img src='dbimg/fleshbar.png' height='16' width='%0.2f' ></td>
			<td width='26' rowspan='11'><img src='dbimg/SATscoresD.gif'></td>
			<td class='right' align='left'><img src='dbimg/orangebar.png' height='16' width='%0.2f' ></td>
			<td width='15' rowspan='11'><img src='dbimg/SATscoresF.gif'></td>
		      </tr>
        ''' % (s[0][0],s[1][1])
    for i in range(1,6):
        rv=rv + '''<tr><td width='120'><img src='dbimg/SATscoresG.gif'></td>
			<td width='120'><img src='dbimg/SATscoresG.gif'></td>
		      </tr>
		      <tr>
			<td class='left' align='right'><img src='dbimg/fleshbar.png' height='16' width='%0.2f' ></td>
			<td class='right' align='left'><img src='dbimg/orangebar.png' height='16' width='%0.2f' ></td>
		      </tr>''' % (s[0][i],s[1][i])
    return rv + "		      <tr>			<td colspan='5'><img src='dbimg/SATscoresH.gif'></td>		      </tr></table>"
actRange=[k for k in ['6~11','12~17','18~23','24~29','30~36'][::-1]]
def pACT(s):
    return '\t'.join([s[0][i].replace('\t','').replace('\n','') for i in range(5)]) +'\t' + '\t'.join([s[1][i].replace('\t','').replace('\n','') for i in range(5)])
    rv = '''		    <table cellspacing='0' cellpadding='0' align='center'>
		      <tr><td border="1px">math</td><td>range</td><td>English</td></tr>'''
    for i in range(5):
        rv=rv + '''<tr>
		        <td style="border-style:solid;border-width:1px" border="1px" align='right'>%0.2f</td>
                        <td style="border-style:solid;border-width:1px" border="1px" align='center'>%s</td>
			<td style="border-style:solid;border-width:1px" border="1px" align='left'>%0.2f</td>
		      </tr>
        ''' % (s[0][i],actRange[i],s[1][i])
    return rv + "</table>"
    rv='''		    <table class='sat' cellspacing='0' cellpadding='0' align='center'>
		      <tr><td colspan='5'><img src='dbimg/ACTscoresA.gif'></td></tr>
		      <tr><td width='14' rowspan='11'><img src='dbimg/SATscoresB.gif'></td>
		        <td class='left' align='right'><img src='dbimg/fleshbar.png' height='16' width='%0.2f' ></td>
			<td width='26' rowspan='11'><img src='dbimg/SATscoresD.gif'></td>
			<td class='right' align='left'><img src='dbimg/orangebar.png' height='16' width='%0.2f' ></td>
			<td width='15' rowspan='11'><img src='dbimg/SATscoresF.gif'></td>
		      </tr>
        ''' % (s[0][0],s[1][1])
    for i in range(1,6):
        rv=rv + '''<tr><td width='120'><img src='dbimg/SATscoresG.gif'></td>
			<td width='120'><img src='dbimg/SATscoresG.gif'></td>
		      </tr>
		      <tr>
			<td class='left' align='right'><img src='dbimg/fleshbar.png' height='16' width='%0.2f' ></td>
			<td class='right' align='left'><img src='dbimg/orangebar.png' height='16' width='%0.2f' ></td>
		      </tr>''' % (s[0][i],s[1][i])
    return rv + "		      <tr>			<td colspan='5'><img src='dbimg/SATscoresH.gif'></td>		      </tr></table>"
    
    

pfuncs=[pName,
        pRank,
        pAddr,
        pSAT,
        pACT,
        pRcmd,
        pInfo,
        pMap,
        pSports,
        pESL,
        pIssue,
        #pGPA,
        pTese,]
def insertInfo(uInfo):
    global f
    for index,key in enumerate(keys):
        f.write('\t%s' % pfuncs[index](uInfo[key]))
    f.write('\n')
    f.flush()
    return
    f.write('\n<tr>')
    for index,key in enumerate(keys):
        f.write('\n<td style="border-style:solid;border-width:1px;">%s</td>' % pfuncs[index](uInfo[key]))
    f.write('\n</tr>')
    f.flush()

def main():
    global addr
    init()
    for state in states:
        addr = state
        getStateImg(state)
        universities = getUniversities(state)
        for university in universities:
            uInfo = getUniversityInfo(university)
            insertInfo(uInfo)

if __name__ == '__main__':
    main()
